""" AnyBackup is a windows specific backup program to keep multiple volumes in sync with multiple volumes
    Copyright (C) 2011  Andrew Hopkinson

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Exported Classes:

    backupManager -- the backup class for AnyBackup, it contains the drive objects, meta data, and all the backup/restore logic
    """


import ConfigParser
from sql import SQLLite
from enum import AEnums
import DriveUtil
import sys
import re
import shutil
import os
import math
import Pyro.core
import config
import stat
import hashlib
import traceback
import wx
import logging
from time import sleep
from mixins import messagingMixin, pathMixin
from operator import attrgetter

messagingQueue = None
errorQueue = None
errorAnswerQueue = None

logger = logging.getLogger('AnyBackup')

import time

def print_timing(func):
    def wrapper(*arg):
        t1 = time.time()
        res = func(*arg)
        t2 = time.time()
        print '%s took %0.3f ms' % (func.func_name, (t2-t1)*1000.0)
        return res
    return wrapper

def file_cmp(a,b):
    return cmp(a[AEnums.PREFIX].lower(),b[AEnums.PREFIX].lower()) or cmp(a[AEnums.PATH].lower(),b[AEnums.PATH].lower()) or cmp(a[AEnums.NAME].lower(),b[AEnums.NAME].lower())

class backupManager(messagingMixin):
    """ the backup class for AnyBackup, it contains the drive objects, meta data and all the backup/restore logic """
    BACKUP=2
    CONTENT=3
    ALL=4
    DUPLICATE=8
    UNNAMED=7
    INVALIDCONFIG=6
    MISSING=9
    BALANCED='balanced'
    STICKY='sticky'
    version = '0.9.7'

    def __init__(self,setName=None):
        self._setName = setName
        self._skipList = []
        self._validExt = []
        self._pendingWriteDirs = {}
        self._pendingWrite = {}
        self.r = SQLLite()
        self.r.resetCache()
        if not self._setName:
            self._setName = self.r.getSets()[0]
        self.readConfigs()

    def addSet(self,set):
        self.r.addSet(set)

    def removeSet(self,set):
        self.r.removeSet(set)

    def getSetName(self):
        try:
            return self._setName
        except AttributeError:
            self._setName = 'AnyBackup'
            return self._setName

    def setName(self,setName):
        self._setName = setName

    def setErrorQueue(self,queue):
        global errorQueue
        errorQueue = queue

    def isBackupMode(self,drives):
        if drives:
            for driveRef in drives:
                if driveRef[AEnums.TYPE] != AEnums.BACKUP:
                    return False
            return True
        return False

    def setErrorAnswerQueue(self,queue):
        global errorAnswerQueue
        errorAnswerQueue = queue

    def sendError(self,error):
        if errorQueue:
            errorQueue.put(error,1)
        else:
            query = raw_input('{0} not connected, connect and proceed? (y/n):'.format(error))
            if query == 'y':
                self.answer = True
            else:
                self.answer = False

    def getErrorAnswer(self):
        if not errorAnswerQueue:
            try:
                return self.answer
            except AttributeError:
                return False
        else:
            while errorAnswerQueue.empty():
                sleep(1)
            return errorAnswerQueue.get_nowait()

    def setMessagingQueue(self,queue):
        global messagingQueue
        messagingQueue = queue
        for drive in self._backupDrives + self._contentDrives:
            drive.setMessagingQueue(queue)

    def toggleDrive(self,driveRef):
        if driveRef.isBackup():
            self._backupDrives.remove(driveRef)
            self._contentDrives.append(driveRef)
            driveRef.setIsBackup(False)
        else:
            self._contentDrives.remove(driveRef)
            self._backupDrives.append(driveRef)
            driveRef.setIsBackup(True)
        return True

    def readConfigs(self):
        config = ConfigParser.RawConfigParser()
        try:
            config.read('AnyBackup.cfg')
            backupMode = config.get('Backup','drive-selection')
            if backupMode == self.BALANCED or backupMode == self.STICKY:
                self.backupMode = backupMode
            else:
                self.backupMode = self.BALANCED
                return self.INVALIDCONFIG
        except:
            self.backupMode = self.BALANCED
            return self.INVALIDCONFIG
        return True

    def getBackupMode(self):
        return self.backupMode

    def sortFiles(self,files):
        files.sort(file_cmp)
        return files

    def getRootFile(self,driveRef):
        return self.r.getRootFile(driveRef)

    def getFiles(self,fileRef,dirs=False):
        if AEnums.ID in fileRef:
            return self.r.getFiles(fileRef[AEnums.ID],dirs)
        elif type(fileRef) in [str,unicode]:
            return self.r.getFiles(fileRef,dirs)
        else:
            return self.r.getRootFile(fileRef,dirs)

    def getFileName(self,fileRef):
        return fileRef[AEnums.NAME]

    def getDir(self,fileRef):
        return fileRef[AEnums.PATH]

    def isDir(self,fileRef):
        return fileRef[AEnums.DIR]

    def getDrive(self,fileRef):
        return self.r.getDriveByFile(fileRef)

    def remoteIndexDrive(self,drive):
        if not AEnums.REMOTEHOST in drive and AEnums.REMOTEPATH in drive:
            return False
        try:
            self.r.removeFiles([drive])
            commDict = {}
            commDict[AEnums.SEP] = '\\'
            commDict[AEnums.NAME] = drive[AEnums.NAME]
            commDict[AEnums.SERIAL] = drive[AEnums.SERIAL]
            commDict[AEnums.PATHS] = self.r.getPaths(drive)
            commDict[AEnums.REMOTEPATH] = drive[AEnums.REMOTEPATH]
            commDict[AEnums.SKIPLIST] = self.r.getSkipList(self._setName)
            commDict[AEnums.VALIDEXTS] = self.r.getValidExt(self._setName)
            commDict[AEnums.SETNAME] = drive[AEnums.SETNAME]
            server = Pyro.core.getProxyForURI("PYROLOC://%s:%d/indexserver"%(drive[AEnums.REMOTEHOST],config.pyro_port+1))
            result = server.index(commDict)
            for file in result[AEnums.FILES]:
                self.r.addFile(drive,file,file[AEnums.ID] in result[AEnums.ROOT])
            self.r.updateFreeSpace(drive)
            self.r.resetCache()
            return result
        except:
            raise
            return False

    def refreshDrive(self,driveRef):
        return self.indexDrives([driveRef])

    def refreshDrives(self,type):
        if type is None:
            type = backupManager.ALL
        drives = []
        if type == backupManager.ALL or type == backupManager.BACKUP:
            drives += self.getBackupDrives()
        if type == backupManager.ALL or type == backupManager.CONTENT:
            drives += self.getContentDrives()
        self.indexDrives(drives)

    def getSkipList(self):
        return self.r.getSkipList(self._setName)

    def addSkip(self, skip):
        if skip:
            self.r.addSkip(skip,self._setName)
            logger.info('Add to skip list: {0}'.format(skip))

    def setSkipList(self,skip):
        if skip:
            self.r.setSkip(skip,self._setName)

    def getValidExt(self):
        return self.r.getValidExt(self._setName)

    def addValidExt(self,ext):
        if ext:
            self.r.addValidExt(ext,self._setName)

    def setValidExt(self,ext):
        if ext:
            self.r.setValidExt(ext,self._setName)

    def addBackupDrive(self,letter,index=True):
        return self.addDrive(letter,AEnums.BACKUP,index)

    def addContentDrive(self,letter,paths,index=True):
        return self.addDrive(letter,AEnums.CONTENT,index,paths)

    def removeDrives(self,drivesIn):
        if not drivesIn:
            logger.error('Undefined drive reference, check your selection')
            return False
        for d in drivesIn:
            self.r.removeDrive(d)
        self.r.resetCache()

    def getBackupDrives(self):
        return self.r.getDrives(AEnums.BACKUP,self._setName)

    def getContentDrives(self):
        return self.r.getDrives(AEnums.CONTENT,self._setName)

    def toggleLock(self,drives):
        allLocked = True
        for driveRef in drives:
            if not self.r.isLocked(driveRef):
                allLocked = False
                break
        self.r.lockDrives(drives,not allLocked)

    def searchFiles(self,regex):
        return self.r.search(regex,self._setName)

    def uniqueCheck(self,name,serial):
        for d in self.getContentDrives() + self.getBackupDrives():
            if d[AEnums.NAME] == name and d[AEnums.SERIAL] == serial:
                logger.error("You've already added " + name + " - " + serial + "!")
                return False
            elif d[AEnums.NAME] == name:
                logger.warn("WARNING: " + name + " -- You've got two drives with the same Volume Name -- this is a bad idea! (Repent!)")
        return True

    def addDrive(self,letter,type,index=True,paths=None):
        name = DriveUtil.getName(letter)
        serial = DriveUtil.getSerial(letter)
        logger.info('Adding drive {0}:'.format(letter))
        if not DriveUtil.exists(letter):
            logger.error('Drive {0}:\\ does not exist!'.format(letter))
            return AEnums.MISSING
        if not name:
            logger.error('Drive {0}:\\ must be named before adding!'.format(letter))
            return AEnums.UNNAMED
        if type == AEnums.BACKUP:
            p = self.createPath(letter+":",self._setName)
            if not os.path.exists(p):
                os.mkdir(p)
        if name and serial:
            if not self.uniqueCheck(name,serial):
                logger.error('Drive {0}:\\ has already been added!'.format(letter))
                return AEnums.DUPLICATE
            d = {
                AEnums.NAME:name,
                AEnums.SERIAL:serial,
                AEnums.TYPE:type,
                AEnums.SETNAME:self._setName,
            }
            self.r.addDrive(d,paths=paths)
            if index:
                self.indexDrives([d])
            logger.info('Drive {0}:\\ added'.format(letter))
            self.r.updateFreeSpace(d)
            return True
        return False

    def isConnected(self,drive):
        if AEnums.NAME in d and AEnums.SERIAL in drive:
            return DriveUtil.getLetter(drive[AEnums.NAME],drive[AEnums.SERIAL])

    def createPath(self,*args):
        pathStr = u''
        for piece in args:
                if pathStr:
                        pathStr += os.sep
                pathStr += piece
        while '\\\\' in pathStr:
                pathStr = pathStr.replace('\\\\','\\')
        return pathStr

    def toFullPath(self,path):
        path = path.replace('\\\\','\\')
        return u'\\\\?\\'+path

    def fullPath(self,*args):
        return u'\\\\?\\'+self.createPath(*args)

    def cleanUp(self,d):
        backupLetter = DriveUtil.getLetter(d[AEnums.NAME],d[AEnums.SERIAL])
        if not backupLetter:
            return
        dir = self.createPath(backupLetter+":\\AnyBackup")
        for file in os.listdir(unicode(dir)):
            path = self.createPath(dir,file)
            if os.path.isdir(self.toFullPath(path)):
                self.cleanDir(path)

    def cleanDir(self,path):
        files = os.listdir(self.toFullPath(path))
        if not files:
            try:
                os.chmod(self.toFullPath(path),stat.S_IWRITE)
                os.rmdir(self.toFullPath(path))
            except Exception, e:
                logger.error('Could not remove empty path {0}: {1}'.format(path.encode('utf-16'),str(e)))
            return
        for file in files:
            filepath = self.createPath(path,file)
            if os.path.isdir(self.toFullPath(filepath)):
                self.cleanDir(filepath)
        if not os.listdir(self.toFullPath(path)):
            self.cleanDir(path)

    def indexDrives(self,drives):
        for d in drives:
            self.r.removeFiles([d])
            letter = DriveUtil.getLetter(d[AEnums.NAME],d[AEnums.SERIAL])
            if letter:
                self.fileCount = 0
                validext = self.r.getValidExt(self._setName)
                if validext:
                    validext = '|'.join(validext)
                    validext = "\.(" + validext + ")$"
                else:
                    validext = None
                skip = '|'.join(self.r.getSkipList(self._setName))
                if skip:
                    skip = re.compile(str(skip),re.IGNORECASE)
                else:
                    skip = None
                if d[AEnums.TYPE] == AEnums.BACKUP:
                    self.cleanUp(d)
                    file = self.index("",d,letter,validext,skip,root=True)
                    #self.r.setRoot(d,[x[AEnums.ID] for x in [self.index("",d,letter,validext,skip,root=True)]])
                    if file:
                        self.r.addFile(d,file,root=True)
                else:
                    dirList = self.r.getPaths(d)
                    if dirList > 1 or (dirList == 1 and dirlist != os.sep):
                        files = []
                        for path in dirList:
                            files.append(self.index(path,d,letter,validext,skip,root=True))
                        #self.r.setRoot(d,[x[AEnums.ID] for x in files])
                        if files:
                            [self.r.addFile(d,x,root=True) for x in files]
                    else:
                        file = self.index(dirList[0],d,letter,validext,skip,root=True)
                        #self.r.setRoot(d,file)
                        if file:
                            self.r.addFile(d,file,root=True)
            else:
                return AEnums.MISSING
            self.r.commit()
            self.r.resetCache()
            return True

    def setMessagingQueue(self,queue):
        global messagingQueue
        messagingQueue = queue

    def sendMessage(self,message):
        if messagingQueue:
                messagingQueue.put(message,1)
        else:
                print message

    def index(self,path,drive,letter,validext,skip,root=False):
        if drive[AEnums.TYPE] == AEnums.BACKUP:
            dir = self.createPath(letter+":",self._setName,path)
        else:
            dir = self.createPath(letter+":",path)
        logger.debug('Indexing: {0}'.format(dir.encode('utf-8')))
        while dir.endswith(os.sep):
            dir = dir.rstrip(os.sep)
        if not os.path.isdir(dir):
            logger.warn("Can only call index on a directory!")
            return None
        if dir.endswith(':'):
            dir += os.sep
        try:
            files = os.listdir(unicode(dir))
        except:
            raise
            return None
        fd = {
            AEnums.NAME : os.path.basename(dir),
            AEnums.PATH : os.path.dirname(path),
            AEnums.PREFIX : drive[AEnums.NAME]+':',
            AEnums.DIR : True,
            AEnums.VOLNAME : drive[AEnums.NAME],
            AEnums.SERIAL : drive[AEnums.SERIAL],
            AEnums.SIZE : 0,
            AEnums.MTIME : 0,
            AEnums.PARENT : '',
            AEnums.SETNAME : self._setName,
        }
        m = hashlib.md5()
        m.update(fd[AEnums.NAME].encode('UTF-16'))
        m.update(fd[AEnums.PATH].encode('UTF-16'))
        m.update(fd[AEnums.VOLNAME])
        m.update(fd[AEnums.SERIAL])
        m.update(fd[AEnums.SETNAME])
        if root:
            m.update(AEnums.ROOT)
        id = m.hexdigest()
        fd[AEnums.ID] = id
        passedFiles = []
        if validext:
            ext = re.compile(validext,re.IGNORECASE)
        else:
            ext = None
        for foundFile in files[:]:
            if not os.access(self.createPath(dir,foundFile),os.R_OK):
                continue
            if skip:
                if re.search(skip,foundFile):
                    continue
            if not os.path.isdir(self.createPath(dir,foundFile)):
                if ext:
                    if not re.search(ext,foundFile):
                        continue
                self.fileCount += 1
                self.sendMessage("Refreshing " + drive[AEnums.NAME] + "-" + drive[AEnums.SERIAL] + ' - Found %i file(s)... %s'%(self.fileCount,foundFile))
            newFile = None
            if os.path.isdir(self.createPath(dir,foundFile)):
                newFile = self.index(self.createPath(path,foundFile),drive,letter,validext,skip)
            else:
                mtime = int(math.floor(os.path.getmtime(self.fullPath(dir,foundFile))))
                if mtime%2:
                    mtime += 1
                newFile = {
                    AEnums.NAME : foundFile,
                    AEnums.PATH : path,
                    AEnums.PREFIX : drive[AEnums.NAME]+':',
                    AEnums.DIR : False,
                    AEnums.SIZE : float(os.path.getsize(self.fullPath(dir,foundFile)))/float(1024),
                    AEnums.MTIME : mtime,
                    AEnums.VOLNAME : drive[AEnums.NAME],
                    AEnums.SERIAL : drive[AEnums.SERIAL],
                    AEnums.SETNAME : self._setName,
                }
                m = hashlib.md5()
                m.update(newFile[AEnums.NAME].encode('UTF-16'))
                m.update(newFile[AEnums.PATH].encode('UTF-16'))
                m.update(newFile[AEnums.VOLNAME])
                m.update(newFile[AEnums.SERIAL])
                m.update(newFile[AEnums.SETNAME])
                id = m.hexdigest()
                newFile[AEnums.ID] = id
            if newFile:
                newFile[AEnums.PARENT] = fd[AEnums.ID]
                passedFiles.append(newFile)
        if not passedFiles and not root:
            return None
        for tmp in passedFiles:
            self.r.addFile(drive,tmp)
        return fd

    def getFile(self,file):
        return self.r.getFile(file)

    def remoteReady(self,d):
        if d.get(AEnums.REMOTEHOST,None) and d.get(AEnums.REMOTEPATH,None):
            return True
        else:
            return False

    def getFullPath(self,fileRef):
        return self.r.getFullPath(fileRef)

    def getSets(self):
        return self.r.getSets()

    @print_timing
    def hashify(self,drive,skiplist):
        hash = {}
        for fileRef in self.r.getDriveFiles(drive):
            if fileRef[AEnums.DIR]:
                continue
            key = self.getKey(fileRef)
            if key in hash:
                hash[key],oldFile=self.chooseFileCopy(hash[key],fileRef)
            else:
                hash[key]=fileRef
        return hash

    def hashifyDuplicates(self,drive,skiplist,hash=None):
        dupes = []
        if not hash:
            hash = {}
        for fileRef in self.r.getDriveFiles(drive):
            if fileRef[AEnums.DIR]:
                continue
            key = self.getKey(fileRef)
            if key in hash:
                hash[key],dupe=self.chooseFileCopy(hash[key],fileRef)
                dupes.append(dupe)
            else:
                hash[key]=fileRef
        return hash,dupes

    def chooseFileCopy(self,file1,file2):
        if self.r.isLocked(self.getBackupDrive(file1)):
            return file2,file1
        else:
            return file1,file2

    def escapeString(self,str):
        str = re.sub("\?","",str)
        str = re.sub("\+","",str)
        str = re.sub("\[","",str)
        str = re.sub("\-","",str)
        str = re.sub("\&","",str)
        str = re.sub("\]","",str)
        str = re.sub("\(","",str)
        str = re.sub("\)","",str)
        str = re.sub("\{","",str)
        str = re.sub("\}","",str)
        while re.search(r'^%s'%re.escape(os.sep),str,re.IGNORECASE):
            str = re.sub(r'^%s'%re.escape(os.sep),"",str)
        return str.lower()

    def getKey(self,fileRef):
        m = hashlib.md5()
        m.update(fileRef[AEnums.PATH].replace('{0}{1}'.format(os.sep,os.sep),os.sep).strip(os.sep).encode('UTF-16'))
        m.update(fileRef[AEnums.NAME].encode('UTF-16'))
        m.update(str(fileRef[AEnums.MTIME]))
        m.update(str(fileRef[AEnums.SIZE]))
        return m.hexdigest()

    @print_timing
    def reduceHash(self,drive,hash):
        for fileRef in self.r.getDriveFiles(drive):
            key = self.getKey(fileRef)
            if key in hash:
                if int(hash[key][AEnums.MTIME]) != int(fileRef[AEnums.MTIME]) or hash[key][AEnums.SIZE] != fileRef[AEnums.SIZE]:
                    continue
                del hash[key]
        return hash

    def listNewFiles(self,fileRef):
        output = True

    @print_timing
    def hashDrives(self,toHash,toReduce):
        fileHash = {}
        files = []
        skipList = '|'.join(self._skipList)
        for driveRef in toHash:
            fileHash.update(self.hashify(driveRef,skipList))
        for driveRef in toReduce:
            fileHash = self.reduceHash(driveRef,fileHash)
        files = fileHash.values()
        files = self.sortFiles(files)
        return files

    def getDuplicates(self):
        allDuplicates = []
        fileHash = {}
        skipList = '|'.join(self.r.getSkipList(self._setName))
        for driveRef in self.getBackupDrives():
            if self.r.getRootFile(driveRef):
                fileHash, duplicates = self.hashifyDuplicates(driveRef,skipList,fileHash)
                if duplicates:
                    allDuplicates.extend(duplicates)
        return allDuplicates

    def getNewFiles(self):
        files = self.hashDrives(self.getContentDrives(),self.getBackupDrives())
        return files

    def getOldFiles(self):
        files = self.hashDrives(self.getBackupDrives(),self.getContentDrives())
        return files

    def getBackupDrive(self,fileRef):
        if self.backupMode == self.BALANCED:
            return self.getMostFreeDrive(self.BACKUP)
        else:
            for driveRef in self.getBackupDrives():
                if self.r.findDir(driveRef,fileRef[AEnums.PATH]):
                    return driveRef
        return None

    def getContentDrive(self,fileRef):
        for driveRef in self.getContentDrives():
            if self.r.findDir(driveRef,fileRef[AEnums.PATH]):
                return driveRef
        return None

    def getMostFreeDrive(self,driveType):
        if not driveType:
            driveType = backupManager.BACKUP
        mostFree = 0
        mostDrive = None
        drives = []
        if driveType == backupManager.BACKUP:
            drives = self.getBackupDrives()
        elif driveType == backupManager.CONTENT:
            drives = self.getContentDrives()
        for driveRef in drives:
            d = driveRef
            if d[AEnums.TYPE] == AEnums.BACKUP:
                if AEnums.LOCKED in d:
                    continue
            freeSpace = int(d[AEnums.FREESPACE])*1024 - self.getPendingWrite(driveRef)
            if freeSpace > mostFree:
                mostDrive = driveRef
                mostFree = freeSpace
        return mostDrive

    def getPendingWrite(self,drive):
        return self._pendingWrite.get(drive[AEnums.NAME]+drive[AEnums.SERIAL],0)

    def addPendingWrite(self,drive,fileDict):
        key = drive[AEnums.NAME]+drive[AEnums.SERIAL]
        if self._pendingWrite.get(key,0)+fileDict[AEnums.SIZE] > drive[AEnums.FREESPACE]*1048576:
            return False
        else:
            self._pendingWriteDirs.get(key,[]).append(fileDict[AEnums.DIR])
            self._pendingWrite[key] = self._pendingWrite.get(key,0) +fileDict[AEnums.SIZE]
            return True

    def addPendingDelete(self,drive,fileRef):
        key = drive[AEnums.NAME]+drive[AEnums.SERIAL]
        if key not in self._pendingWrite:
            self._pendingWrite[key] = 0
        self._pendingWrite[key] -= fileRef[AEnums.SIZE]

    def clearPendingWrite(self,drive=None):
        if drive:
            drives = [drive]
        else:
            drives = self.getBackupDrives() + self.getContentDrives()
        for drive in drives:
            key = drive[AEnums.NAME]+drive[AEnums.SERIAL]
            self._pendingWriteDirs[key] = []
            self._pendingWrite[key] = 0

    def isLocked(self,drive):
        return self.r.isLocked(drive)

    def getDriveById(self,drive):
        return self.r.getDrive(drive)

    def copyTo(self,files,path):
        sorted = {}
        errors = []
        for fileRef in files:
            k = (fileRef[AEnums.VOLNAME],fileRef[AEnums.SERIAL])
            if k not in sorted:
                sorted[k] = []
            sorted[k].append(fileRef)
        for key in sorted:
            letter = DriveUtil.getLetter(*key)
            d = self.r.getDrive(*key+(self._setName,))
            while not letter:
                self.sendError(key[0]+"-"+key[1])
                answer = self.getErrorAnswer()
                if answer:
                    letter = DriveUtil.getLetter(*key)
                else:
                    letter = False
                    break
            if not letter:
                errors.append('Cancelled copy from {0}'.format(key[0]+"-"+key[1]))
                continue
            for fpath in self.r.getFullPaths(sorted[key],type=d[AEnums.TYPE]):
                try:
                    shutil.copy2(fpath,path)
                except:
                    logger.error(traceback.format_exc())
                    print traceback.format_exc()
                    errors.append('Error copying file {0}'.format(fpath))
        return errors

    def backupNewFiles(self):
        files = self.getNewFiles()
        toRemove = self.getOldFiles()+self.getDuplicates()
        toStore = {}
        totalRemove = len(toRemove)
        msg = "Sorting files..."
        self.sendMessage(msg)
        logger.info(msg)
        missingDrives = {}
        drivesToRefresh = []
        errorMessages = []
        total = len(files)
        total += totalRemove
        count = 1
        removeCount = 1
        self.clearPendingWrite()
        for removeFile in toRemove:
            buDrive = self.getDrive(removeFile)
            key = buDrive[AEnums.NAME]+buDrive[AEnums.SERIAL]
            if not buDrive:
                continue
            if self.r.isLocked(buDrive):
                error = "%s is locked, cannot remove %s!"%(buDrive,removeFile[AEnums.NAME])
                errorMessages.append(error)
                logger.error(error)
                continue
            self.addPendingDelete(buDrive,removeFile)
            if key not in toStore:
                toStore[key]={}
                toStore[key]['old']=[]
                toStore[key]['files']=[]
                toStore[key]['drive']=buDrive
            toStore[key]['old'].append(removeFile)
        for fileRef in files:
            buDrive = self.getBackupDrive(fileRef)
            d = buDrive
            key = buDrive[AEnums.NAME] + buDrive[AEnums.SERIAL]
            if buDrive:
                if not self.addPendingWrite(buDrive,fileRef):
                    buDrive = None
            if not buDrive:
                buDrive = self.getMostFreeDrive(backupManager.BACKUP)
            if buDrive:
                if not self.addPendingWrite(buDrive,fileRef):
                    error = "Not enough space to store "+fileRef[AEnums.NAME]+" on any backup drive, consider adding another backup drive? Unlock backup drives with free space?"
                    errorMessages.append(error)
                    logger.error(error)
                    continue
            if not buDrive:
                error = "Error getting an open drive, no backup drives? All backup drives full? Locked?"
                errorMessages.append(error)
                logger.error(error)
                return errorMessages
            if self.r.isLocked(buDrive):
                error = "%s is locked, cannot backup %s!"%(d[AEnums.NAME],fileRef[AEnums.NAME])
                errorMessages.append(error)
                logger.error(error)
                continue
            if key not in toStore:
                toStore[key]={}
                toStore[key]['files']=[]
                toStore[key]['drive']=buDrive
            toStore[key]['files'].append(fileRef)
        for id in toStore.keys():
            buRef = toStore[id]['drive']
            backupLetter = None
            backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME],buRef[AEnums.SERIAL])
            old = []
            if 'old' in toStore[id]:
                old = toStore[id]['old']
            for fileRef in old:
                msg = "Removing old file " + fileRef[AEnums.NAME] + " on " + buRef[AEnums.NAME] + "... File " + str(count) + "/" + str(total)
                self.sendMessage(msg)
                logger.info(msg)
                if missingDrives.has_key(id):
                    error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" not connected, skipping " + fileRef[AEnums.NAME]
                    errorMessages.append(error)
                    logger.error(error)
                    continue
                while not backupLetter:
                    self.sendError(buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL])
                    answer = self.getErrorAnswer()
                    if answer:
                        backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME],buRef[AEnums.SERIAL])
                    else:
                        missingDrives[buRef[AEnums.NAME]+buRef[AEnums.SERIAL]] = id
                        backupLetter = False
                        break
                if not backupLetter:
                    error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" not connected, skipping all files for this drive."
                    errorMessages.append(error)
                    logger.error(error)
                    break
                buName = buRef[AEnums.NAME]
                toRemove = self.fullPath(backupLetter+":\\",buRef[AEnums.SETNAME],fileRef[AEnums.PATH],fileRef[AEnums.NAME])
                try:
                    os.chmod(toRemove,stat.S_IWRITE)
                    if os.path.exists(toRemove):
                        if os.path.isdir(toRemove):
                            shutil.rmtree(toRemove)
                        else:
                            os.unlink(toRemove)
                except Exception, e:
                    error = "Error deleting " + toRemove + ": %s" % (str(e))
                    errorMessages.append(error)
                    logger.error(error)
                count += 1
                self.r.updateFreeSpace(buRef)
            msg = "Backing up new files on " + buRef[AEnums.NAME] + "..."
            self.sendMessage(msg)
            logger.info(msg)
            fileList = []
            if 'files' in toStore[id]:
                fileList = toStore[id]['files']
            for fileRef in fileList:
                msg = "Backing up " + fileRef[AEnums.NAME] + " to " + buRef[AEnums.NAME] + "... File " + str(count) + "/" + str(total)
                self.sendMessage(msg)
                logger.info(msg)
                sourceSerial = fileRef[AEnums.SERIAL]
                sourceName = fileRef[AEnums.VOLNAME]
                sourceLetter = DriveUtil.getLetter(sourceName,sourceSerial)
                if not sourceLetter:
                    error = sourceName+"-"+sourceSerial+" missing, skipping " + fileRef[AEnums.NAME]
                    errorMessages.append(error)
                    logger.error(error)
                    continue
                if missingDrives.has_key(buRef[AEnums.NAME]+buRef[AEnums.SERIAL]):
                    error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" not connected, skipping "+fileRef[AEnums.NAME]
                    errorMessages.append(error)
                    logger.error(error)
                    continue
                while not backupLetter:
                    self.sendError(buDrive[AEnums.NAME]+"-"+buDrive[AEnums.SERIAL])
                    answer = self.getErrorAnswer()
                    if answer == wx.OK:
                        backupLetter = DriveUtil.getLetter(buDrive[AEnums.NAME],buDrive[AEnums.SERIAL])
                    else:
                        missingDrives[id] = id
                        backupLetter = False
                        break
                if not backupLetter:
                    error = buDrive[AEnums.NAME]+"-"+buDrive[AEnums.SERIAL]+" not connected, skipping all files for this drive."
                    errorMessages.append(error)
                    logger.error(error)
                    break
                try:
                    source = self.fullPath(sourceLetter+":",fileRef[AEnums.PATH],fileRef[AEnums.NAME])
                    targetDir = self.createPath(backupLetter+":",buRef[AEnums.SETNAME],fileRef[AEnums.PATH])
                except UnicodeDecodeError, e:
                    error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" can't copy source due to naming issue, skipping: %s" % e
                    errorMessages.append(error)
                    logger.error(error)
                    continue
                if not os.path.exists(self.toFullPath(targetDir)):
                    try:
                        os.makedirs(targetDir)
                    except:
                        error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" can't create dir, skipping "+fileRef[AEnums.NAME]
                        errorMessages.append(error)
                        logger.error(error)
                        continue
                try:
                    shutil.copy2(source,self.fullPath(targetDir,fileRef[AEnums.NAME]))
                except IOError, e:
                    error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" can't copy source, skipping " + fileRef[AEnums.NAME]+": %s" % e
                    errorMessages.append(error)
                    logger.error(error)
                    continue
                buName = buRef[AEnums.NAME]
                count += 1
                self.r.updateFreeSpace(buRef)
            msg = "Refreshing " + buRef[AEnums.NAME] + "-" + buRef[AEnums.SERIAL] + "..."
            self.sendMessage(msg)
            logger.info(msg)
            self.refreshDrive(buRef)
        return errorMessages

    def restoreOldFiles(self):
        toRestore = self.getOldFiles()
        toStore = {}
        total = len(toRestore)
        msg = "Sorting files..."
        self.sendMessage(msg)
        logger.info(msg)
        missingDrives = {}
        drivesToRefresh = {}
        errorMessages = []
        count = 1
        for restoreFile in toRestore:
            buDrive = self.getDrive(restoreFile)
            key = buDrive[AEnums.NAME] + buDrive[AEnums.SERIAL]
            if not buDrive:
                continue
            if not toStore.has_key(key):
                toStore[key]={}
                toStore[key]['files']=[]
                toStore[key]['drive']=buDrive
            toStore[key]['files'].append(restoreFile)
        for id in toStore.keys():
            buRef = toStore[id]['drive']
            msg = "Restoring files from " + buRef[AEnums.NAME] + "..."
            self.sendMessage(msg)
            logger.info(msg)
            backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME],buRef[AEnums.SERIAL])
            for fileRef in toStore[id]['files']:
                mostFree = False
                contentRef = self.getContentDrive(fileRef)
                if not contentRef:
                    mostFree = True
                    contentRef = self.getMostFreeDrive(backupManager.CONTENT)
                d = contentRef
                if not self.addPendingWrite(contentRef,fileRef):
                    if not mostFree:
                        contentRef = self.getMostFreeDrive(backupManager.CONTENT)
                    if not self.addPendingWrite(contentRef,fileRef):
                        error = "Cannot find a content drive with enough space to store " + fileRef[AEnums.NAME] + ", all drives full?"
                        errorMessages.append(error)
                        logger.error(error)
                        continue
                contentLetter = DriveUtil.getLetter(d[AEnums.NAME],d[AEnums.SERIAL])
                if not contentLetter:
                    error = d[AEnums.NAME]+"-"+d[AEnums.SERIAL]+" not connected, skipping "+fileRef[AEnums.NAME]
                    errorMessages.append(error)
                    logger.error(error)
                msg = "Restoring "+fileRef[AEnums.NAME]+" to "+d[AEnums.NAME]+"... File "+str(count)+"/"+str(total)
                self.sendMessage(msg)
                logger.info(msg)
                if missingDrives.has_key(buRef[AEnums.NAME]+buRef[AEnums.SERIAL]):
                    errorMessages.append(buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL])
                while backupLetter is None:
                    self.sendError(d[AEnums.NAME]+"-"+d[AEnums.SERIAL])
                    answer = self.getErrorAnswer()
                    if answer:
                        backupLetter = DriveUtil.getLetter(buRef[AEnums.NAME],buRef[AEnums.SERIAL])
                    else:
                        missingDrives[buRef[AEnums.ID]] = buRef[AEnums.ID]
                        backupLetter = False
                        break
                if not backupLetter:
                    error = buRef[AEnums.NAME]+"-"+buRef[AEnums.SERIAL]+" not connected, skpping restore for all missing files from this drive."
                    errorMessages.append(error)
                    logger.error(error)
                    break
                buName = buRef[AEnums.NAME]
                source = self.fullPath(backupLetter+":",buRef[AEnums.SETNAME],fileRef[AEnums.PATH],fileRef[AEnums.NAME])
                targetDir = self.createPath(contentLetter+":",fileRef[AEnums.PATH])
                if not os.path.exists(self.toFullPath(targetDir)):
                    try:
                        os.makedirs(targetDir)
                    except:
                        error = d[AEnums.NAME]+"-"+d[AEnums.SERIAL]+" can't create dir, skipping " + fileRef[AEnums.NAME]
                        errorMessages.append(error)
                        logger.error(error)
                        continue
                try:
                    shutil.copy2(source,self.fullPath(targetDir,fileRef[AEnums.NAME]))
                except:
                    error = d[AEnums.NAME]+"-"+d[AEnums.SERIAL]+" can't copy source, skpping "+fileRef[AEnums.NAME]
                    logger.error(error)
                    errorMessages.append(error)
                    continue
                count += 1
                self.r.updateFreeSpace(contentRef)
                drivesToRefresh[id]=contentRef

        for d in drivesToRefresh.values():
            msg = "Refreshing "+d[AEnums.NAME]+"-"+d[AEnums.SERIAL]+"..."
            self.sendMessage(msg)
            logger.info(msg)
            self.refreshDrive(d)
        return errorMessages

    def refresh(self,drives,remote=False):
        errorMessages = []
        for d in drives:
            proceed = False
            if not d[AEnums.TYPE] == AEnums.BACKUP:
                if remote and AEnums.REMOTEHOST in d and AEnums.REMOTEPATH in d:
                    msg = 'Remote indexing {0}-{1}'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                    logger.info(msg)
                    self.sendMessage(msg)
                    result = self.remoteIndexDrive(d)
                    if not result:
                        error = 'Error remote indexing {0}-{1}, unknown error'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                        logger.error(error)
                        self.sendMessage(error)
                        errorMessages.append(error)
                    elif result == AEnums.NOTEXIST:
                        error = 'Remote path for {0}-{1} does not exist'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                        logger.error(error)
                        self.sendMessage(error)
                        errorMessages.append(error)
                    else:
                        msg = 'Remote indexing done for {0}-{1}'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                        logger.info(msg)
                        self.sendMessage(msg)
                else:
                    proceed = True
            else:
                proceed = True
            if not remote or proceed:
                msg = 'Refreshing {0}-{1}'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                logger.info(msg)
                self.sendMessage(msg)
                result = self.refreshDrive(d)
                while result == AEnums.MISSING:
                    self.sendError(d[AEnums.NAME]+"-"+d[AEnums.SERIAL])
                    answer = self.getErrorAnswer()
                    if answer:
                        result = self.refreshDrive(d)
                    else:
                        error = 'Could not refresh {0}-{1}, drive is not connected'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                        logger.error(error)
                        self.sendMessage(error)
                        errorMessages.append(error)
                        result = 13
                        break
                if result == 13:
                    continue
                if not result:
                    error = 'Could not refresh {0}-{1}, unknown error'.format(d[AEnums.NAME],d[AEnums.SERIAL])
                    logger.error(error)
                    self.sendMessage(error)
                else:
                    msg = "Refreshed " + d[AEnums.NAME] + "-" + d[AEnums.SERIAL] + "!"
                    logger.info(msg)
                    self.sendMessage(msg)
        return errorMessages